弹力设计篇之"熔断设计" - [2026重制版]

resilience4j:
  circuitbreaker:
    instances:
      # 核心支付服务的熔断器（严格配置）
      paymentService:
        slidingWindowType: COUNT_BASED       # 基于调用次数统计
        slidingWindowSize: 20                # 最近20次调用
        minimumNumberOfCalls: 5              # 至少5次调用才开始计算
        failureRateThreshold: 50             # 失败率>=50%触发熔断
        slowCallRateThreshold: 80            # 慢调用率>=80%
        slowCallDurationThreshold: 3000ms    # >3s算慢调用
        permittedNumberOfCallsInHalfOpenState: 3   # 半开放3个探测请求
        maxWaitDurationInHalfOpenState: 0         # 半开状态无额外等待
        waitDurationInOpenState: 30s         # 熔断持续30秒
        automaticTransitionFromOpenToHalfOpenEnabled: true  # 自动转半开
        registerHealthIndicator: true        # 注册健康检查端点
        
      # 外部API服务的熔断器（宽松配置）
      externalApi:
        slidingWindowType: TIME_BASED        # 基于时间窗口统计
        slidingWindowSize: 60s               # 60秒窗口
        minimumNumberOfCalls: 10             # 至少10次调用
        failureRateThreshold: 60             # 60%失败率才熔断
        slowCallDurationThreshold: 5000ms    # >5s算慢调用
        waitDurationInOpenState: 15s         # 熔断15秒
        permittedNumberOfCallsInHalfOpenState: 5  # 半开放5个请求
        
      # 数据库访问的熔断器（快速失败）
      databaseAccess:
        slidingWindowType: COUNT_BASED
        slidingWindowSize: 10                # 较小窗口
        failureRateThreshold: 70             # 更高的阈值
        slowCallDurationThreshold: 1000ms   # 数据库查询>1s就算慢
        waitDurationInOpenState: 10s         # 短暂熔断
        permittedNumberOfCallsInHalfOpenState: 1   # 只放行1个探测请求

@Service
@RequiredArgsConstructor
@Slf4j
public class PaymentServiceWithCircuitBreaker {

    private final PaymentGatewayClient gatewayClient;
    private final CircuitBreakerRegistry cbRegistry;
    private final MeterRegistry meterRegistry;

    /**
     * 方式一：注解方式（推荐）
     */
    @CircuitBreaker(
        name = "paymentService",
        fallbackMethod = "fallbackProcessPayment"
    )
    public PaymentResult processPayment(PaymentRequest request) {
        log.info("Processing payment via circuit breaker");
        return gatewayClient.charge(request);
    }

    /**
     * 熔断开启时的降级方法
     * 
     * 注意：
     * 1. 参数列表要在原方法参数后面加上Exception
     * 2. 返回类型要一致
     * 3. 可以在这里实现降级逻辑（如走备用通道、返回缓存数据等）
     */
    public PaymentResult fallbackProcessPayment(PaymentRequest request, CallNotPermittedException e) {
        log.warn("Circuit breaker is OPEN for payment service! Falling back...");
        
        // 降级策略选项：
        // 1. 返回默认值/空结果
        // 2. 从缓存读取旧数据
        // 3. 切换到备用支付渠道
        // 4. 提示用户稍后重试
        
        return PaymentResult.degraded(
            "Payment service temporarily unavailable, please try later",
            false  // 不扣款
        );
    }

    /**
     * 方式二：编程式方式（更灵活的状态判断和监控）
     */
    public PaymentResult processPaymentProgrammatic(PaymentRequest request) {
        CircuitBreaker circuitBreaker = cbRegistry.circuitbreaker("paymentService");

        // 注册自定义事件监听器（用于监控和告警）
        circuitBreaker.getEventPublisher()
            .onStateTransition(event -> {
                log.info("Circuit breaker state transition: {} -> {}", 
                         event.getStateTransition().getFromState(),
                         event.getStateTransition().getToState());
                
                // 发送告警通知
                if (event.getStateTransition().getToState() == CircuitBreaker.State.OPEN) {
                    alertService.sendAlert(AlertLevel.WARNING,
                        "Circuit breaker OPEN for payment service",
                        "Failure rate exceeded threshold");
                }
            })
            .onError(event -> {
                log.debug("Circuit breaker recorded error: duration={}ms, result={}",
                         event.getElapsedDuration().toMillis(),
                         event.getThrowable().getMessage());
            })
            .onSuccess(event -> {
                if (event.getElapsedDuration().toMillis() > 1000) {
                    log.warn("Slow success call detected: duration={}ms",
                             event.getElapsedDuration().toMillis());
                }
            });

        // 使用Supplier包装调用
        Supplier<PaymentResult> supplier = CircuitBreaker.decorateSupplier(circuitBreaker, () -> {
            return gatewayClient.charge(request);
        });

        try {
            return supplier.get();
        } catch (CallNotPermittedException e) {
            // 熔断器打开时的特殊处理
            log.error("Circuit breaker is OPEN, call not permitted", e);

            // 记录指标到Prometheus
            Counter.builder("circuit_breaker_calls_rejected")
                .tag("service", "payment")
                .register(meterRegistry)
                .increment();

            return PaymentResult.rejected(e.getMessage());
        }
    }
}

# istio-circuit-breaker.yaml
apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata:
  name: payment-service-cb
  namespace: production
spec:
  host: payment-service.production.svc.cluster.local
  trafficPolicy:
    connectionPool:
      tcp:
        maxConnections: 100              # TCP最大连接数
        connectTimeout: 2s               # 连接超时
        tcpKeepalive:
          time: 7200s                   # keepalive间隔
          interval: 75s
          probes: 3                     # 保活探针次数
      http:
        http1MaxPendingRequests: 50     # HTTP/1.1 最大等待请求数
        http2MaxRequests: 1000          # HTTP/2 最大并发请求数
        idleTimeout: 300s               # 空闲超时
        requestTimeout: 10s             # 单请求超时
        maxRequestsPerConnection: 10    # 每连接最大请求数
        h2UpgradePolicy: DEFAULT        # HTTP/2升级策略
    outlierDetection:  # ⭐ 异常实例检测（即熔断）
      consecutive5xxErrors: 5           # 连续5个5xx错误
      consecutiveGatewayErrors: 3      # 连续3个网关错误（502/503/504）
      interval: 30s                    # 检查间隔（扫描周期）
      baseEjectionTime: 30s             # 基础驱逐时间（被移除至少30秒）
      maxEjectionPercent: 50            # 最大驱逐比例（最多驱逐50%的Pod）
      minHealthPercent: 50              # 最小健康实例比例（低于此值停止驱逐）
      consecutiveLocalOriginFailure: 5 # 本地连续失败数
---
# 可选：配合VirtualService使用
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: payment-service-vs
  namespace: production
spec:
  hosts:
  - payment-service
  http:
  - match:
    - uri:
        prefix: /api/payment
    retries:
      attempts: 3                       # 重试次数
      perTryTimeout: 2s                 # 每次重试超时
      retryOn: gateway-error,connect-failure,refused-stream,retriable-status-code
      retriableStatusCodes:
      - 503                             # 仅对503重试
    timeout: 10s                        # 总超时时间
    route:
    - destination:
        host: payment-service
        subset: v1
    fault:  # 故障注入（用于混沌工程测试）
      abort:
        percentage:
          value: 1                      # 1%的概率
        httpStatus: 500                 # 返回500错误
      delay:
        percentage:
          value: 5                      # 5%的概率
        fixedDelay: 2s                  # 延迟2秒

@Configuration
public class CircuitBreakerMetricsConfig {

    @Bean
    public Customizer<CircuitBreakerCustomizer> circuitBreakerCustomizer(MeterRegistry registry) {
        return customizer -> customizer
            // 注册所有熔断器指标到Prometheus
            .circuitBreakerConfig(CircuitBreakerConfig.ofDefaults())
            // 自定义标签
            .tags(
                "environment", "production",
                "team", "payment"
            )
            // 注册事件发布器
            .registerEventConsumer(
                event -> switch (event.getEventType()) {
                    case SUCCESS -> Counter.builder("cb_success")
                            .tag("name", event.getCircuitBreakerName())
                            .register(registry)
                            .increment();
                    case ERROR -> Counter.builder("cb_error")
                            .tag("name", event.getCircuitBreakerName())
                            .tag("error_class", event.getThrowable().getClass().getSimpleName())
                            .register(registry)
                            .increment();
                    case NOT_PERMITTED -> Counter.builder("cb_rejected")
                            .tag("name", event.getCircuitBreakerName())
                            .register(registry)
                            .increment();
                    case STATE_TRANSITION -> Gauge.builder("cb_state")
                            .tag("name", event.getCircuitBreakerName())
                            .tag("state", event.getStateTransition().getToState().name())
                            .register(registry,
                                () -> event.getStateTransition().getToState() == CircuitBreaker.State.OPEN ? 1 : 0
                            );
                    default -> {}
                }
            );
    }
}